# -*- coding: utf-8 -*-

# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://doc.scrapy.org/en/latest/topics/item-pipeline.html
import sys
reload(sys)
sys.setdefaultencoding('utf-8')
from scrapy.exceptions import DropItem
from spider.items import H6080Item, H6080MovieInfo
import MySQLdb
from pybloom import BloomFilter


class Spider1Pipeline(object):
    def __init__(self):
        #self.ids_seen = []
        self.ids_seen = BloomFilter(capacity=5000000, error_rate=0.001)
        #self.movienames_seen = []
        self.movienames_seen = BloomFilter(capacity=5000000, error_rate=0.001)
        self.conn = MySQLdb.connect(host='localhost', port=3306, user='root', passwd='ROOT', db='movie-website1', charset='utf8')
        self.cur = self.conn.cursor()
        self.cur.execute('SELECT url FROM spider_h6080')
        result = self.cur.fetchall()
        for row in result:
            self.ids_seen.add(row[0])
        self.cur.execute('SELECT picurl FROM spider_h6080_movieinfo')
        result = self.cur.fetchall()
        for row in result:
            self.movienames_seen.add(row[0])

    def process_item(self, item, spider):
        print "Deep of ids_seen: %s" % self.ids_seen.count
        print "Deep of movienames_seen: %s" % self.movienames_seen.count
        if isinstance(item, H6080Item):
            if item['url'] in self.ids_seen:
                raise DropItem("Exist Exception! Duplicate item found: %s" % item['name'])
            else:
                sql = ("INSERT INTO spider_h6080 (NAME,url,num) VALUES ('%s', '%s', '%s')" % (item['name'], item['url'], item['num']))
                try:
                    self.cur.execute(sql)
                    self.conn.commit()
                    self.ids_seen.add(item['url'])
                except Exception as err:
                    raise DropItem("DB Exception! Duplicate item found: %s" % err)
                return item
        elif isinstance(item, H6080MovieInfo):
            if item['picurl'] in self.movienames_seen:
                raise DropItem("Exist Exception! Duplicate item found: %s" % item['name'])
            sql = "INSERT INTO spider_h6080_movieinfo (moviename,actor,TYPES,AREA,publishtime,countnumber,introduce,director,picurl) VALUES ('%s', '%s', '%s', '%s','%s', '%s', '%s', '%s', '%s')" % (item['name'], item['actor'], item['types'], item['area'], item['publishtime'], item['countnumber'], item['introduce'], item['director'], item['picurl'])
            try:
                self.cur.execute(sql)
                self.conn.commit()
                self.movienames_seen.add(item['picurl'])
            except Exception as err:
                raise DropItem("DB Exception! Duplicate item found: %s" % err)
            return item
        else:
            pass
